package com.erik.ipmonitor.parse;

import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.alibaba.excel.EasyExcel;
import com.erik.ipmonitor.model.excel.LinkExcel;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;

public class HtmlLinkChecker {
    // 分离不同级别的线程池，避免相互阻塞
    private static ExecutorService fileExecutor;
    private static ExecutorService pageExecutor;
    private static ExecutorService linkExecutor;

    // 线程池配置参数
    private static final int CPU_COUNT = Runtime.getRuntime().availableProcessors();
    private static final int FILE_POOL_SIZE = Math.min(CPU_COUNT, 4); // 文件处理线程池大小
    private static final int PAGE_POOL_SIZE = CPU_COUNT * 2;          // 页面处理线程池大小
    private static final int LINK_POOL_SIZE = CPU_COUNT * 4;          // 链接检查线程池大小
    private static final long KEEP_ALIVE_TIME = 30;
    private static final TimeUnit TIME_UNIT = TimeUnit.SECONDS;

    // 超时设置
    private static final long FILE_PROCESS_TIMEOUT = 30;    // 文件处理超时(分钟)
    private static final long PAGE_PROCESS_TIMEOUT = 5;     // 页面处理超时(分钟)
    private static final long LINK_CHECK_TIMEOUT = 30;      // 链接检查超时(秒)
    private static final int NETWORK_TIMEOUT = 3000;        // 网络请求超时(毫秒)

    public static void main(String[] args) {
        long startTime = System.currentTimeMillis();
        System.out.println("程序开始执行，时间: " + new Date());

        // 初始化不同级别的线程池
        initExecutors();

        String filePath = "/Users/erik.zhou/Desktop/www/temp/it";
        List<File> allFiles = getAllFiles(new File(filePath));
        System.out.println("发现 " + allFiles.size() + " 个文件需要处理");

        try {
            // 并行处理所有文件
            List<Future<List<LinkExcel>>> fileFutures = new ArrayList<>();
            for (File file : allFiles) {
                fileFutures.add(fileExecutor.submit(() -> processFile(file)));
            }

            // 收集所有结果，带超时控制
            List<LinkExcel> allLinkExcels = new ArrayList<>();
            for (int i = 0; i < fileFutures.size(); i++) {
                Future<List<LinkExcel>> future = fileFutures.get(i);
                File file = allFiles.get(i);
                try {
                    // 每个文件处理设置超时
                    List<LinkExcel> results = future.get(FILE_PROCESS_TIMEOUT, TimeUnit.MINUTES);
                    allLinkExcels.addAll(results);
                    System.out.println("已完成 " + (i + 1) + "/" + allFiles.size() + " 个文件处理");
                } catch (TimeoutException e) {
                    System.err.println("文件处理超时: " + file.getAbsolutePath() + "，已跳过");
                    future.cancel(true); // 取消超时的任务
                } catch (InterruptedException | ExecutionException e) {
                    System.err.println("处理文件 " + file.getAbsolutePath() + " 时发生错误: " + e.getMessage());
                }
            }

            // 写入Excel结果
            if (!allLinkExcels.isEmpty()) {
                String excelFileName = "link_results_" + System.currentTimeMillis() + ".xlsx";
                EasyExcel.write(excelFileName, LinkExcel.class)
                        .sheet("link_status")
                        .doWrite(allLinkExcels);
                System.out.println("所有结果已写入 " + excelFileName + "，共 " + allLinkExcels.size() + " 条记录");
            } else {
                System.out.println("没有发现需要记录的链接状态");
            }

        } finally {
            // 优雅关闭所有线程池
            shutdownExecutor(fileExecutor, "fileExecutor");
            shutdownExecutor(pageExecutor, "pageExecutor");
            shutdownExecutor(linkExecutor, "linkExecutor");

            long endTime = System.currentTimeMillis();
            System.out.println("程序执行结束，总耗时: " + (endTime - startTime) / 1000 + "秒");
        }
    }

    /**
     * 初始化所有线程池
     */
    private static void initExecutors() {
        fileExecutor = new ThreadPoolExecutor(
                FILE_POOL_SIZE,
                FILE_POOL_SIZE,
                KEEP_ALIVE_TIME,
                TIME_UNIT,
                new LinkedBlockingQueue<>(100),
                new ThreadFactory() {
                    private int count = 0;
                    @Override
                    public Thread newThread(Runnable r) {
                        Thread thread = new Thread(r);
                        thread.setName("file-thread-" + (++count));
                        thread.setDaemon(false);
                        return thread;
                    }
                },
                new ThreadPoolExecutor.CallerRunsPolicy()
        );

        pageExecutor = new ThreadPoolExecutor(
                PAGE_POOL_SIZE,
                PAGE_POOL_SIZE,
                KEEP_ALIVE_TIME,
                TIME_UNIT,
                new LinkedBlockingQueue<>(1000),
                new ThreadFactory() {
                    private int count = 0;
                    @Override
                    public Thread newThread(Runnable r) {
                        Thread thread = new Thread(r);
                        thread.setName("page-thread-" + (++count));
                        thread.setDaemon(false);
                        return thread;
                    }
                },
                new ThreadPoolExecutor.CallerRunsPolicy()
        );

        linkExecutor = new ThreadPoolExecutor(
                LINK_POOL_SIZE,
                LINK_POOL_SIZE,
                KEEP_ALIVE_TIME,
                TIME_UNIT,
                new LinkedBlockingQueue<>(5000),
                new ThreadFactory() {
                    private int count = 0;
                    @Override
                    public Thread newThread(Runnable r) {
                        Thread thread = new Thread(r);
                        thread.setName("link-thread-" + (++count));
                        thread.setDaemon(false);
                        return thread;
                    }
                },
                new ThreadPoolExecutor.CallerRunsPolicy()
        );
    }

    /**
     * 优雅关闭线程池
     */
    private static void shutdownExecutor(ExecutorService executor, String name) {
        if (executor != null) {
            executor.shutdown();
            try {
                if (!executor.awaitTermination(1, TimeUnit.MINUTES)) {
                    System.out.println("线程池 " + name + " 强制关闭中...");
                    executor.shutdownNow();
                }
            } catch (InterruptedException e) {
                executor.shutdownNow();
            }
            System.out.println("线程池 " + name + " 已关闭");
        }
    }

    /**
     * 处理单个文件的任务
     */
    private static List<LinkExcel> processFile(File file) {
        String filePath = file.getAbsolutePath();
        System.out.println("开始处理文件: " + filePath + " (线程: " + Thread.currentThread().getName() + ")");

        try {
            // 1. 从sitemap提取所有页面URL
            List<String> pageUrls = extractUrlsFromSitemap(filePath);
            // 提取类型（增强容错性）
            String type = extractTypeFromFileName(filePath);
            System.out.println("文件 " + filePath + " 提取到 " + pageUrls.size() + " 个页面URL，类型: " + type);

            if (pageUrls.isEmpty()) {
                return Collections.emptyList();
            }

            // 2. 并行分析每个页面中的链接
            List<Callable<List<LinkExcel>>> pageTasks = pageUrls.stream()
                    .map(pageUrl -> (Callable<List<LinkExcel>>) () -> processSinglePage(pageUrl, type))
                    .collect(Collectors.toList());

            // 执行所有页面任务并获取结果，带超时控制
            List<Future<List<LinkExcel>>> pageFutures = pageExecutor.invokeAll(pageTasks,
                    PAGE_PROCESS_TIMEOUT, TimeUnit.MINUTES);

            // 合并所有页面的结果
            List<LinkExcel> fileResults = new ArrayList<>();
            for (Future<List<LinkExcel>> future : pageFutures) {
                if (future.isCancelled()) {
                    System.err.println("页面处理任务已超时取消");
                    continue;
                }

                try {
                    fileResults.addAll(future.get());
                } catch (ExecutionException e) {
                    System.err.println("处理页面时发生错误: " + e.getMessage());
                }
            }

            System.out.println("文件 " + filePath + " 处理完成，发现 " + fileResults.size() + " 个需要记录的链接");
            return fileResults;

        } catch (Exception e) {
            System.err.println("处理文件 " + filePath + " 时发生错误: " + e.getMessage());
            e.printStackTrace();
            return Collections.emptyList();
        }
    }

    /**
     * 处理单个页面，提取并检查链接
     */
    private static List<LinkExcel> processSinglePage(String pageUrl, String type) {
        List<LinkExcel> results = new ArrayList<>();

        try {
            System.out.println("处理页面: " + pageUrl + " (线程: " + Thread.currentThread().getName() + ")");

            // 获取页面中的所有链接，带超时控制
            List<String> links = extractLinksFromPage(pageUrl);
            if (links.isEmpty()) {
                System.out.println("页面 " + pageUrl + " 没有找到符合条件的链接");
                return results;
            }

            System.out.println("页面 " + pageUrl + " 找到 " + links.size() + " 个链接，开始检查状态");

            // 并发检查链接状态
            List<Callable<LinkStatus>> linkTasks = links.stream()
                    .map(link -> (Callable<LinkStatus>) () -> checkLinkStatus(link))
                    .collect(Collectors.toList());

            // 链接检查带超时控制
            List<Future<LinkStatus>> linkFutures = linkExecutor.invokeAll(linkTasks,
                    LINK_CHECK_TIMEOUT, TimeUnit.SECONDS);

            // 收集检查结果
            int timeoutCount = 0;
            for (Future<LinkStatus> future : linkFutures) {
                if (future.isCancelled()) {
                    timeoutCount++;
                    continue;
                }

                try {
                    LinkStatus status = future.get();
                    if (status.code != 200) {
                        LinkExcel excel = new LinkExcel();
                        excel.setLinkType(type);
                        excel.setErrorLink(status.url);
                        excel.setStatus(status.code);
                        excel.setQuoteLink(pageUrl);
                        results.add(excel);
                    }
                } catch (ExecutionException e) {
                    System.err.println("检查链接状态时发生错误: " + e.getMessage());
                }
            }

            if (timeoutCount > 0) {
                System.out.println("页面 " + pageUrl + " 有 " + timeoutCount + " 个链接检查超时");
            }

            System.out.println("页面 " + pageUrl + " 处理完成，发现 " + results.size() + " 个需要记录的链接");

        } catch (Exception e) {
            System.err.println("处理页面 " + pageUrl + " 时发生错误: " + e.getMessage());
        }

        return results;
    }

    /**
     * 从文件名提取类型（增强容错性）
     */
    private static String extractTypeFromFileName(String filePath) {
        try {
            int lastUnderscore = filePath.lastIndexOf("_");
            int lastDot = filePath.lastIndexOf(".");

            if (lastUnderscore != -1 && lastDot != -1 && lastUnderscore < lastDot) {
                return filePath.substring(lastUnderscore + 1, lastDot);
            }
            return "unknown";
        } catch (Exception e) {
            return "unknown";
        }
    }

    /**
     * 从sitemap提取URL
     */
    private static List<String> extractUrlsFromSitemap(String filePath) throws Exception {
        List<String> urls = new ArrayList<>();

        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        // 增加XML解析的容错性
        factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
        factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
        factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);

        DocumentBuilder builder = factory.newDocumentBuilder();
        org.w3c.dom.Document document = builder.parse(filePath);
        document.getDocumentElement().normalize();

        NodeList urlList = document.getElementsByTagName("url");

        for (int i = 0; i < urlList.getLength(); i++) {
            Node urlNode = urlList.item(i);
            if (urlNode.getNodeType() == Node.ELEMENT_NODE) {
                org.w3c.dom.Element urlElement = (org.w3c.dom.Element) urlNode;
                NodeList locList = urlElement.getElementsByTagName("loc");

                if (locList.getLength() > 0) {
                    String loc = locList.item(0).getTextContent();
                    urls.add(loc);
                }
            }
        }

        return urls;
    }

    /**
     * 从网页中提取所有链接
     */
    private static List<String> extractLinksFromPage(String pageUrl) {
        List<String> links = new ArrayList<>();

        try {
            // 连接网页并获取HTML，设置超时
            Document doc = Jsoup.connect(pageUrl)
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
                    .timeout(NETWORK_TIMEOUT)
                    .followRedirects(true)
                    .get();

            // 提取所有a标签的href属性
            Elements anchors = doc.select("a[href]");
            for (Element anchor : anchors) {
                // 获取绝对URL（处理相对路径）
                String link = anchor.absUrl("href");

                // 过滤出社区相关的链接
                if (link.startsWith("https://community.fs.com/")) {
                    links.add(link);
                }
            }

        } catch (IOException e) {
            System.err.println("无法获取页面内容: " + pageUrl + " - " + e.getMessage());
        }

        return links;
    }

    /**
     * 检查单个链接的状态码，增加超时和重试机制
     */
    private static LinkStatus checkLinkStatus(String url) {
        int retryCount = 0;
        System.out.println("检查链接: " + url);
        while (retryCount < 2) { // 最多重试1次
            try {
                HttpResponse accept = HttpRequest.head(url)
                        .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
                        .header("Accept", "*/*")
                        .timeout(5000) // 超时时间（毫秒）
                        .execute();
                Integer responseCode = accept.getStatus();
                if (responseCode.equals(301)) {
                    String redirectUrl = accept.header("Location");
                    if (redirectUrl.contains("404")) {
                        return new LinkStatus(url, 404);
                    }
                }
                return new LinkStatus(url, responseCode);
            } catch (Exception e) {
                retryCount++;
                if (retryCount >= 2) {
                    // 重试后仍然失败，返回错误状态
                    return new LinkStatus(url, -1);
                }
                // 重试前短暂等待
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException ie) {
                    Thread.currentThread().interrupt();
                    return new LinkStatus(url, -1);
                }
            }
        }
        return new LinkStatus(url, -1);
    }

    /**
     * 递归获取指定目录下的所有文件，增加过滤非XML文件的逻辑
     */
    public static List<File> getAllFiles(File directory) {
        List<File> fileList = new ArrayList<>();
        // 检查目录是否存在
        if (!directory.exists()) {
            System.out.println("目录不存在: " + directory.getAbsolutePath());
            return fileList;
        }

        // 检查是否是目录
        if (!directory.isDirectory()) {
            System.out.println("不是一个目录: " + directory.getAbsolutePath());
            return fileList;
        }

        // 获取目录下的所有文件和子目录
        File[] files = directory.listFiles();
        if (files != null) {
            for (File file : files) {
                if (file.isDirectory()) {
                    // 如果是目录，递归处理
                    fileList.addAll(getAllFiles(file));
                } else {
                    // 只处理XML文件
                    if (file.getName().toLowerCase().endsWith(".xml")) {
                        fileList.add(file);
                    } else {
                        System.out.println("跳过非XML文件: " + file.getAbsolutePath());
                    }
                }
            }
        }

        return fileList;
    }

    /**
     * 存储链接及其状态码的内部类
     */
    private static class LinkStatus {
        String url;
        int code;

        LinkStatus(String url, int code) {
            this.url = url;
            this.code = code;
        }
    }
}
